#!/usr/bin/env python
# Create: 2018/10/25
__author__ = '749B'
"""tornado"""

from bs4 import BeautifulSoup
from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import ioloop

url_list = [
    'https://github.com/explore',
    'https://www.djangoproject.com/',
    'http://www.python-requests.org/en/master/',
    'https://jquery.com/',
    'https://getbootstrap.com/',
    'https://www.solarwinds.com/',
    'https://www.zabbix.com/',
    'http://open-falcon.org/',
    'https://www.python.org/',
    'http://www.jetbrains.com/',
]


def asynchronous_fetch():
    http_client = AsyncHTTPClient()

    # 创建一个函数内的函数，来处理返回的结果
    def handle_response(response):
        """
        处理返回值内容（需要维护计数器，来停止IO循环），调用 ioloop.IOLoop.current().stop()
        :param response:
        :return:
        """
        if response.error:
            print("Error:", response.error)
        else:
            # print(response.headers)
            # print(response.body)
            soup = BeautifulSoup(response.body, features='html.parser')
            title = soup.find('title')
            print(title)
        # 自己加的停止的方法，实现方法可能不是很正规
        # print(response.effective_url)
        curr_url = response.effective_url
        if curr_url in url_list:
            url_list.remove(curr_url)
        if not url_list:
            ioloop.IOLoop.current().stop()

    for url in url_list:
        # 异步处理结束后会调用指定的callback的函数
        http_client.fetch(HTTPRequest(url), callback=handle_response)
        # 下面这句和上面效果一样，模块内部会判断参数的isinstance是否是HTTPRequest
        # 如果不是则，HTTPRequest(url, **kwargs)
        # 这里的**kwargs，就是如果要给请求加任何参数，就用关键参数传参
        # http_client.fetch(url, callback=handle_response)


if __name__ == '__main__':
    ioloop.IOLoop.current().add_callback(asynchronous_fetch)
    ioloop.IOLoop.current().start()


